package xyz.zinglizingli.books.schedule;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import xyz.zinglizingli.books.RestTemplateUtil;
import xyz.zinglizingli.books.po.Book;
import xyz.zinglizingli.books.po.BookContent;
import xyz.zinglizingli.books.po.BookIndex;
import xyz.zinglizingli.books.service.BookService;

import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Service
public class CrawlBooksSchedule {

    @Autowired
    private BookService bookService;

    @Value("${mysql.charset}")
    private String mysqlCharset;

    @Value("${books.lowestScore}")
    private Float lowestScore;

    RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");


    //插入或更新
    @Scheduled(fixedRate = 1000 * 60 * 60 * 24)
    public void crawBquge11Books() {
        RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");

        final String baseUrl = "https://m.biqudao.com";
        System.out.println("开始时间" + new Date());


        for (int i = 1; i <= 7; i++) {

            int finalI = i;
            new Thread(
                    () -> {

                        try {
                            //拼接分类URL
                            int page = 1;//起始页码
                            int totalPage = page;
                            String catBookListUrl = baseUrl + "/bqgelhb/" + finalI + "/" + page + ".html";
                            String forObject = getByHttpClient(catBookListUrl);
                            if (forObject != null) {
                                //匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
                                Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
                                Matcher matcher = pattern.matcher(forObject);
                                boolean isFind = matcher.find();
                                System.out.println("匹配分页数" + isFind);
                                if (isFind) {
                                    int currentPage = Integer.parseInt(matcher.group(1));
                                    totalPage = Integer.parseInt(matcher.group(2));
                                    //解析第一页书籍的数据
                                    Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\"");
                                    parseBiquge11Book(bookPatten, forObject, finalI, baseUrl);
                                    while (currentPage < totalPage) {
                                        catBookListUrl = baseUrl + "/bqgelhb/" + finalI + "/" + (currentPage + 1) + ".html";
                                        forObject = getByHttpClient(catBookListUrl);
                                        if (forObject != null) {
                                            //匹配分页数
                                            matcher = pattern.matcher(forObject);
                                            isFind = matcher.find();

                                            if (isFind) {
                                                currentPage = Integer.parseInt(matcher.group(1));
                                                totalPage = Integer.parseInt(matcher.group(2));
                                                parseBiquge11Book(bookPatten, forObject, finalI, baseUrl);
                                            }
                                        } else {
                                            currentPage++;
                                        }
                                    }
                                }
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                        }

                    }
            ).start();


        }

        /*for (int i = 1; i <=56; i++) {

            String bookListUrl = "http://book.suixw.com/modules/article/articlelist.php?class=&page=" + i;

            String forObject = getByHttpClient(bookListUrl, "book.suixw.com");

            if (forObject != null) {
                Pattern bookPatten = Pattern.compile("href=\"http://book.suixw.com/modules/article/articleinfo.php\\?id=(\\d+)\"");
                Matcher bookMatcher = bookPatten.matcher(forObject);
                boolean isFindBook = bookMatcher.find();

                while (isFindBook) {
                    try {
                        long bookNum = Long.parseLong(bookMatcher.group(1));
                        String bookUrl = "http://book.suixw.com/modules/article/articleinfo.php?id=" + bookNum;
                        String forObject1 = getByHttpClient(bookUrl, "book.suixw.com");
                        if (forObject1 != null) {
                            Pattern updateTimePatten = Pattern.compile("最后更新：(\\d+-\\d+-\\d+)");
                            Matcher updateTimeMatch = updateTimePatten.matcher(forObject1);
                            boolean isFindUpdateTime = updateTimeMatch.find();
                            if (isFindUpdateTime) {
                                // Date updateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(updateTimeMatch.group(1)+" "+new SimpleDateFormat("HH:mm:ss").format(new Date()));
                                Date updateTime = new Date();
                                Pattern bookNamePatten = Pattern.compile("<span style=\"font-size:16px; font-weight: bold; line-height: 150%\">([^<]+)</span>");
                                Matcher bookNameMatcher = bookNamePatten.matcher(forObject1);
                                boolean isFindBookName = bookNameMatcher.find();
                                if (isFindBookName) {
                                    String bookName = bookNameMatcher.group(1);
                                    System.out.println(bookName);
                                    Pattern authorPatten = Pattern.compile(">小说作者：([^<]+)<");
                                    Matcher authorMatcher = authorPatten.matcher(forObject1);
                                    boolean isFindAuthor = authorMatcher.find();
                                    if (isFindAuthor) {
                                        String author = authorMatcher.group(1);

                                        Pattern picPtten = Pattern.compile("\"(http://book.suixw.com/files/article/image/\\d+/\\d+/\\d+s\\.jpg)\"");
                                        Matcher picMatcher = picPtten.matcher(forObject1);
                                        if (picMatcher.find()) {
                                            String pic = picMatcher.group(1);

                                            Pattern visitPatten = Pattern.compile(">总点击数：(\\d+)<");
                                            Matcher visitMatcher = visitPatten.matcher(forObject1);
                                            boolean isFindVisit = visitMatcher.find();
                                            if (isFindVisit) {
                                                String visit = visitMatcher.group(1);

                                                Pattern statusPatten = Pattern.compile(">文章状态：([^<]+)<");
                                                Matcher statusMatcher = statusPatten.matcher(forObject1);
                                                boolean isFindStatus = statusMatcher.find();
                                                if (isFindStatus) {
                                                    String status = statusMatcher.group(1);

                                                    String desc = forObject1.substring(forObject1.indexOf("内容简介：") + 5);
                                                    desc = desc.substring(desc.indexOf("&nbsp;&nbsp;&nbsp;&nbsp;") + "&nbsp;&nbsp;&nbsp;&nbsp;".length());
                                                    desc = desc.substring(0, desc.indexOf("<br"));

                                                    Book book = new Book();
                                                    book.setAuthor(author);
                                                    book.setCatid(8);
                                                    book.setBookDesc(desc);
                                                    book.setBookName(bookName);
                                                    DecimalFormat df = new DecimalFormat("#.0");
                                                    float score = Float.parseFloat(df.format((Double.parseDouble(visit) / 1000)));
                                                    book.setScore(score < 7.3f ? 7.3f : score > 10 ? 9.3f : score);
                                                    book.setVisitCount(Long.parseLong(visit));
                                                    book.setPicUrl(pic);
                                                    book.setBookStatus(status);
                                                    book.setUpdateTime(updateTime);

                                                    List<BookIndex> indexList = new ArrayList<>();
                                                    List<BookContent> contentList = new ArrayList<>();

                                                    //读取目录
                                                    String indexUrl = "http://book.suixw.com/modules/article/reader.php?aid=" + bookNum;
                                                    String forObject2 = getByHttpClient(indexUrl, "book.suixw.com");
                                                    if (forObject2 != null) {
                                                        Pattern indexListPatten = Pattern.compile("\"http://book.suixw.com/modules/article/reader.php\\?aid=" + bookNum + "&cid=(\\d+)\">([^<]+)<");
                                                        Matcher indexListMatch = indexListPatten.matcher(forObject2);

                                                        boolean isFindIndex = indexListMatch.find();

                                                        int indexNum = 0;

                                                        //查询该书籍已存在目录号
                                            List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);

                                            while (isFindIndex) {
                                                if (!hasIndexNum.contains(indexNum)) {

                                                                String contentUrl = "http://book.suixw.com/modules/article/reader.php?aid=" + bookNum + "&cid=" + indexListMatch.group(1);
                                                                String indexName = indexListMatch.group(2);


                                                                //查询章节内容
                                                                String forObject3 = getByHttpClient(contentUrl, "book.suixw.com");
                                                                if (forObject3 != null) {
                                                                    String content = forObject3.substring(forObject3.indexOf("<div id=\"content\""));
                                                                    content = content.substring(0, content.indexOf("</div>") + 6);
                                                                    //TODO插入章节目录和章节内容
                                                                    BookIndex bookIndex = new BookIndex();
                                                                    bookIndex.setIndexName(indexName);
                                                                    bookIndex.setIndexNum(indexNum);
                                                                    indexList.add(bookIndex);
                                                                    BookContent bookContent = new BookContent();
                                                                    bookContent.setContent(content);
                                                                    bookContent.setIndexNum(indexNum);
                                                                    contentList.add(bookContent);


                                                                } else {
                                                                    break;
                                                                }

                                                            }

                                                            indexNum++;
                                                            isFindIndex = indexListMatch.find();
                                                        }
                                                        if (indexList.size() == contentList.size() && indexList.size() > 0) {
                                                            bookService.saveBookAndIndexAndContent(book, indexList, contentList);
                                                        }

                                                    }
                                                }

                                            }
                                        }
                                    }
                                }
                            }
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    } finally {
                        isFindBook = bookMatcher.find();
                    }
                }
            }
        }*/

        for (int j = 21; j <= 29; j++) {
            int finalJ = j;
            new Thread(() -> {

                for (int i = 1; i <= 499; i++) {
                    System.out.println("==============分类============：" + finalJ);
                    System.out.println("==============页码============：" + i);
                    int catId = finalJ;
                    int page = i;

                    String bookListUrl = "http://book.sfacg.com/List/default.aspx?&tid=" + catId + "&if=1&PageIndex=" + page;

                    String forObject = getByHttpClient(bookListUrl, "book.sfacg.com");

                    if (forObject != null) {
                        Pattern bookPatten = Pattern.compile("href=\"/Novel/(\\d+)/\"");
                        Matcher bookMatcher = bookPatten.matcher(forObject);
                        boolean isFindBook = bookMatcher.find();

                        while (isFindBook) {
                            try {
                                long bookNum = Long.parseLong(bookMatcher.group(1));
                                String bookUrl = "http://book.sfacg.com/Novel/" + bookNum;
                                String forObject1 = getByHttpClient(bookUrl, "book.sfacg.com");
                                if (forObject1 != null) {
                                    Pattern updateTimePatten = Pattern.compile("更新：(\\d+/\\d+/\\d+ \\d+:\\d+:\\d+)");
                                    Matcher updateTimeMatch = updateTimePatten.matcher(forObject1);
                                    boolean isFindUpdateTime = updateTimeMatch.find();
                                    if (isFindUpdateTime) {
                                        String updateTimeStr = updateTimeMatch.group(1);
                                        String dateStr = updateTimeStr;
                                        int firstPos = dateStr.indexOf("/");
                                        String year = dateStr.substring(0, firstPos);
                                        dateStr = dateStr.substring(firstPos + 1);
                                        firstPos = dateStr.indexOf("/");
                                        String month = dateStr.substring(0, firstPos);
                                        dateStr = dateStr.substring(firstPos + 1);
                                        firstPos = dateStr.indexOf(" ");
                                        String day = dateStr.substring(0, firstPos);
                                        dateStr = dateStr.substring(firstPos + 1);
                                        firstPos = dateStr.indexOf(":");
                                        String hour = dateStr.substring(0, firstPos);
                                        dateStr = dateStr.substring(firstPos + 1);
                                        firstPos = dateStr.indexOf(":");
                                        String minus = dateStr.substring(0, firstPos);
                                        String second = dateStr.substring(firstPos + 1);
                                        if (month.length() == 1) {
                                            month = "0" + month;
                                        }
                                        if (day.length() == 1) {
                                            day = "0" + day;
                                        }
                                        if (hour.length() == 1) {
                                            hour = "0" + hour;
                                        }
                                        if (minus.length() == 1) {
                                            minus = "0" + minus;
                                        }
                                        if (second.length() == 1) {
                                            second = "0" + second;
                                        }


                                        Date updateTime = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").parse(updateTimeStr);


                                        //Date updateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(year+"-"+month+"-"+hour+" "+minus+" "+minus);
                                        Pattern bookNamePatten = Pattern.compile("<h1 class=\"title\">\\s*" +
                                                "<span class=\"text\">([^<]+)</span>\\s*" +
                                                "<span");
                                        Matcher bookNameMatcher = bookNamePatten.matcher(forObject1);
                                        boolean isFindBookName = bookNameMatcher.find();
                                        if (isFindBookName) {
                                            String bookName = bookNameMatcher.group(1);
                                            System.out.println(bookName);
                                            Pattern authorPatten = Pattern.compile("<div class=\"author-name\">\\s*" +
                                                    "<span>([^<]+)</span>\\s*" +
                                                    "</div>");
                                            Matcher authorMatcher = authorPatten.matcher(forObject1);
                                            boolean isFindAuthor = authorMatcher.find();
                                            if (isFindAuthor) {
                                                String author = authorMatcher.group(1);

                                                Pattern picPtten = Pattern.compile("src=\"(http://rs.sfacg.com/web/novel/images/NovelCover/Big/[^\"]+)\"");
                                                Matcher picMatcher = picPtten.matcher(forObject1);
                                                if (picMatcher.find()) {
                                                    String pic = picMatcher.group(1);

                                                    Pattern visitPatten = Pattern.compile(">点击：(\\d+)<");
                                                    Matcher visitMatcher = visitPatten.matcher(forObject1);
                                                    boolean isFindVisit = visitMatcher.find();
                                                    if (isFindVisit) {
                                                        String visit = visitMatcher.group(1);

                                                        Pattern statusPatten = Pattern.compile(">字数：\\d+字\\[([^<]+)\\]<");
                                                        Matcher statusMatcher = statusPatten.matcher(forObject1);
                                                        boolean isFindStatus = statusMatcher.find();
                                                        if (isFindStatus) {
                                                            String status = statusMatcher.group(1);

                                                            if ("已完结".equals(status)) {//先爬已完结的

                                                                status = "已完成";
                                                            }


                                                            Pattern scorePatten = Pattern.compile("<div class=\"num\">\\s*" +
                                                                    "<span>(\\d+\\.\\d+)</span>\\s*" +
                                                                    "</div>");
                                                            Matcher scoreMather = scorePatten.matcher(forObject1);
                                                            boolean isFindScore = scoreMather.find();
                                                            if (isFindScore) {

                                                                float score = Float.parseFloat(scoreMather.group(1));
                                                                //if (score >= 7.0) {

                                                                Pattern descPatten = Pattern.compile("<p class=\"introduce\">\\s*" +
                                                                        "([^<]+)\\s*</p>");
                                                                Matcher descMatcher = descPatten.matcher(forObject1);
                                                                boolean isFindDesc = descMatcher.find();
                                                                if (isFindDesc) {
                                                                    String desc = descMatcher.group(1);

                                                                    Pattern tagPatten = Pattern.compile("<li class=\"tag\">\\s*" +
                                                                            "<a href=\"/stag/\\d+/\" class=\"highlight\"><span class=\"icn\">[^<]+</span><span class=\"text\">([^<]+)</span></a>\\s*" +
                                                                            "</li>");
                                                                    Matcher tagMatch = tagPatten.matcher(forObject1);
                                                                    String tag = "";
                                                                    boolean isFindTag = tagMatch.find();
                                                                    while (isFindTag) {
                                                                        tag += ("," + tagMatch.group(1));
                                                                        isFindTag = tagMatch.find();
                                                                    }

                                                                    if (tag.length() > 0) {
                                                                        tag = tag.substring(1);
                                                                    }


                                                                    Book book = new Book();
                                                                    book.setAuthor(author);
                                                                    book.setCatid(8);
                                                                    book.setBookDesc(desc);
                                                                    book.setBookName(bookName);
                                                                    book.setSoftTag(tag);
                                                                    book.setSoftCat(catId);
                                                                    book.setScore(score>10?8.0f:score);
                                                                    book.setVisitCount(Long.parseLong(visit));
                                                                    book.setPicUrl(pic);
                                                                    book.setBookStatus(status);
                                                                    book.setUpdateTime(updateTime);

                                                                    List<BookIndex> indexList = new ArrayList<>();
                                                                    List<BookContent> contentList = new ArrayList<>();

                                                                    //读取目录
                                                                    String indexUrl = "http://book.sfacg.com/Novel/" + bookNum + "/MainIndex/";
                                                                    String forObject2 = getByHttpClient(indexUrl, "book.sfacg.com");
                                                                    if (forObject2 != null) {
                                                                        Pattern indexListPatten = Pattern.compile("href=\"(/Novel/\\d+/\\d+/\\d+/)\"\\s+title=\"([^\"]+)\\s*");
                                                                        Matcher indexListMatch = indexListPatten.matcher(forObject2);

                                                                        boolean isFindIndex = indexListMatch.find();

                                                                        int indexNum = 0;

                                                                        //查询该书籍已存在目录号
                                                                        List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);

                                                                        while (isFindIndex) {
                                                                            if (!hasIndexNum.contains(indexNum)) {

                                                                                String contentUrl = "http://book.sfacg.com" + indexListMatch.group(1);
                                                                                String indexName = indexListMatch.group(2);


                                                                                //查询章节内容
                                                                                String forObject3 = getByHttpClient(contentUrl, "book.sfacg.co");
                                                                                if (forObject3 != null && !forObject3.contains("内容整改中,请等待")) {
                                                                                    String content = forObject3.substring(forObject3.indexOf("<div class=\"article-content"));
                                                                                    content = content.substring(0, content.indexOf("</div>") + 6);
                                                                                    //TODO插入章节目录和章节内容
                                                                                    BookIndex bookIndex = new BookIndex();
                                                                                    bookIndex.setIndexName(filterEmoji(indexName));
                                                                                    bookIndex.setIndexNum(indexNum);
                                                                                    indexList.add(bookIndex);
                                                                                    BookContent bookContent = new BookContent();
                                                                                    bookContent.setContent(filterEmoji(content));
                                                                                    bookContent.setIndexNum(indexNum);
                                                                                    contentList.add(bookContent);


                                                                                } else {
                                                                                    break;
                                                                                }

                                                                            }

                                                                            indexNum++;
                                                                            isFindIndex = indexListMatch.find();
                                                                        }
                                                                        if (indexList.size() == contentList.size() && indexList.size() > 0) {
                                                                            bookService.saveBookAndIndexAndContent(book, indexList, contentList);
                                                                        }

                                                                    }
                                                                }
                                                            }

                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                            } finally {
                                isFindBook = bookMatcher.find();
                            }
                        }
                    }
                }

            }).start();


        }


    }

    private void parseBiquge11Book(Pattern bookPatten, String forObject, int catNum, String baseUrl) {

        Matcher matcher2 = bookPatten.matcher(forObject);
        boolean isFind = matcher2.find();
        Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
        Matcher scoreMatch = scorePatten.matcher(forObject);
        boolean scoreFind = scoreMatch.find();

        Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
        Matcher bookNameMatch = bookNamePatten.matcher(forObject);
        boolean isBookNameMatch = bookNameMatch.find();

        Pattern authorPatten = Pattern.compile(">作者：([^<]+)<");
        Matcher authoreMatch = authorPatten.matcher(forObject);
        boolean isFindAuthor = authoreMatch.find();


        System.out.println("匹配书籍url" + isFind);

        System.out.println("匹配分数" + scoreFind);
        while (isFind && scoreFind && isBookNameMatch && isFindAuthor) {

            try {
                Float score = Float.parseFloat(scoreMatch.group(1));

                if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
                    Thread.sleep(1000 * 60 * 60 * 24);//因为爬的是龙虎榜，所以遇到第一个8分以下的，之后的都是8分以下的
                    continue;
                }

                String bookName = bookNameMatch.group(1);
                String author = authoreMatch.group(1);
                /*//查询该书籍是否存在
                boolean isExsit = bookService.isExsitBook(bookName, author);
                if (isExsit) {
                    continue;
                }*/

                //System.out.println(new Date()+bookName + "：");

                String bokNum = matcher2.group(1);
                String bookUrl = baseUrl + "/" + bokNum + "/";

                String body = getByHttpClient(bookUrl);
                if (body != null) {
                    Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
                    Matcher statusMatch = statusPatten.matcher(body);
                    if (statusMatch.find()) {
                        String status = statusMatch.group(1);
                        Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
                        Matcher updateTimeMatch = updateTimePatten.matcher(body);
                        if (updateTimeMatch.find()) {
                            String updateTimeStr = updateTimeMatch.group(1);
                            SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
                            Date updateTime = format.parse(updateTimeStr);
                            Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
                            Matcher picMather = picPatten.matcher(body);
                            if (picMather.find()) {
                                String picSrc = picMather.group(1);

                                Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
                                Matcher descMatch = descPatten.matcher(body);
                                if (descMatch.find()) {
                                    String desc = descMatch.group(1);


                                    Book book = new Book();
                                    book.setAuthor(author);
                                    book.setCatid(catNum);
                                    book.setBookDesc(desc);
                                    book.setBookName(bookName);
                                    book.setScore(score>10?8.0f:score);
                                    book.setPicUrl(picSrc);
                                    book.setBookStatus(status);
                                    book.setUpdateTime(updateTime);

                                    List<BookIndex> indexList = new ArrayList<>();
                                    List<BookContent> contentList = new ArrayList<>();

                                    //读取目录
                                    Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/bqge\\d+/all\\.html)\">查看完整目录</a>");
                                    Matcher indexMatch = indexPatten.matcher(body);
                                    if (indexMatch.find()) {
                                        String indexUrl = baseUrl + indexMatch.group(1);
                                        String body2 = getByHttpClient(indexUrl);
                                        if (body2 != null) {
                                            Pattern indexListPatten = Pattern.compile("<a[^/]+style[^/]+href=\"(/bqge\\d+/\\d+\\.html)\">([^/]+)</a>");
                                            Matcher indexListMatch = indexListPatten.matcher(body2);

                                            boolean isFindIndex = indexListMatch.find();

                                            int indexNum = 0;
                                            //查询该书籍已存在目录号
                                            List<Integer> hasIndexNum = bookService.queryIndexCountByBookNameAndBAuthor(bookName, author);

                                            while (isFindIndex) {
                                                if (!hasIndexNum.contains(indexNum)) {

                                                    String contentUrl = baseUrl + indexListMatch.group(1);
                                                    String indexName = indexListMatch.group(2);


                                                    //查询章节内容
                                                    String body3 = getByHttpClient(contentUrl);
                                                    if (body3 != null) {
                                                        Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
                                                        String start = "『章节错误,点此举报』";
                                                        String end = "『加入书签，方便阅读』";
                                                        String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
                                                        //TODO插入章节目录和章节内容
                                                        BookIndex bookIndex = new BookIndex();
                                                        bookIndex.setIndexName(indexName);
                                                        bookIndex.setIndexNum(indexNum);
                                                        indexList.add(bookIndex);
                                                        BookContent bookContent = new BookContent();
                                                        bookContent.setContent(filterEmoji(content));
                                                        bookContent.setIndexNum(indexNum);
                                                        contentList.add(bookContent);
                                                        //System.out.println(indexName);


                                                    } else {
                                                        break;
                                                    }
                                                }
                                                indexNum++;
                                                isFindIndex = indexListMatch.find();
                                            }

                                            if (indexList.size() == contentList.size() && indexList.size() > 0) {
                                                bookService.saveBookAndIndexAndContent(book, indexList, contentList);
                                            }
                                        }

                                    }


                                }


                            }
                        }


                    }

                }

            } catch (Exception e) {

                e.printStackTrace();

            } finally {
                matcher2.find();
                isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
                scoreFind = scoreMatch.find();
                isBookNameMatch = bookNameMatch.find();
                isFindAuthor = authoreMatch.find();
            }


        }

    }

    private String getByHttpClient(String url, String host) {
        try {
            HttpClient httpClient = new DefaultHttpClient();
            // 设置请求和传输超时时间
            /*RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000)
                    .setRedirectsEnabled(false) // 不自动重定向
                    .build();
            HttpGet getReq = new HttpGet(url);
            getReq.setConfig(requestConfig);
           *//* getReq.setHeader("user-agent", "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1");
            getReq.setHeader("Host", host);*//*
            HttpResponse execute = httpClient.execute(getReq);
            if (execute.getStatusLine().getStatusCode() == 200) {
                HttpEntity entity = execute.getEntity();
                return EntityUtils.toString(entity, "gbk");
            } else {
                return null;
            }*/

            ResponseEntity<String> forEntity = restTemplate.getForEntity(url, String.class);
            if (forEntity.getStatusCode() == HttpStatus.OK) {
                return forEntity.getBody();
            } else {
                return null;
            }
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    private String getByHttpClient(String catBookListUrl) {
        try {
            /*HttpClient httpClient = new DefaultHttpClient();
            // 设置请求和传输超时时间
            RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000)
                    .setRedirectsEnabled(false) // 不自动重定向
                    .build();
            HttpGet getReq = new HttpGet(catBookListUrl);
            getReq.setConfig(requestConfig);
            getReq.setHeader("user-agent", "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1");
            HttpResponse execute = httpClient.execute(getReq);
            if (execute.getStatusLine().getStatusCode() == HttpStatus.OK.value()) {
                HttpEntity entity = execute.getEntity();
                return EntityUtils.toString(entity, "utf-8");
            } else {
                return null;
            }*/
            //经测试restTemplate比httpClient效率高出很多倍，所有选择restTemplate
            ResponseEntity<String> forEntity = restTemplate.getForEntity(catBookListUrl, String.class);
            if (forEntity.getStatusCode() == HttpStatus.OK) {
                return forEntity.getBody();
            } else {
                return null;
            }
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }


    public String filterEmoji(String source) {
        if (!"utf8mb4".equals(mysqlCharset) && source != null) {
            Pattern emoji = Pattern.compile("[\ud83c\udc00-\ud83c\udfff]|[\ud83d\udc00-\ud83d\udfff]|[\u2600-\u27ff]", Pattern.UNICODE_CASE | Pattern.CASE_INSENSITIVE);
            Matcher emojiMatcher = emoji.matcher(source);
            if (emojiMatcher.find()) {
                source = emojiMatcher.replaceAll("*");
                return source;
            }
            return source;
        }
        return source;
    }
}
